{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/root/data/train_set.csv',sep='\\t') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2967 6758 339 2021 1854 3731 4109 3792 4149 1519 2058 3912 2465 2410 1219 6654 7539 264 2456 4811 1292 2109 6905 5520 7058 6045 3634 6591 3530 6508 2465 7044 1519 3659 2073 3750 3731 4109 3792 6831 2614 3370 4269 3370 486 5770 4109 4125 3750 5445 2466 6831 6758 3743 3630 1726 2313 5906 826 4516 657 900 1871 7044 3750 2967 3731 1757 1939 648 2828 4704 7039 3706 3750 965 2490 7399 3743 2145 2407 7451 3775 6017 5998 1641 299 4704 2621 7029 3056 6333 433 648 1667 1099 900 2289 1099 648 5780 220 7044 1279 7426 4269 3750 2967 6758 6631 3099 2205 7305 2620 5977 3750 3329 1793 6666 2042 3193 4149 1519 7039 3706 2446 5399 648 4124 2058 3912 248 3193 2252 5649 2212 4939 7239 3310 4525 2400 900 5770 4109 4125 7044 4921 265 1397 4699 1699 669 6407 3750 1271 1271 4741 669 4659 3870 4030 4167 5338 25 3466 6909 4417 1859 3750 1465 7194 648 3938 1571 848 6986 827 2124 3750 1991 7444 7037 2729 908 6308 3750 1889 6810 4190 591 5598 2289 2109 6831 6407 2400 5410 517 900 25 3731 4109 3792 4128 1679 4811 4853 4109 3630 6902 6122 1903 1736 3915 2602 6822 3750 6630 4265 591 729 4448 648 1465 1401 4853 648 5881 6182 4128 1679 4939 2646 652 340 7328 1320 900 1460 619 5505 2376 4853 3272 3750 4853 4109 3630 6902 3362 2810 3750 803 1985 4128 669 19 6508 900 1635 1871 7377 6122 6017 3750 2289 1099 3938 1571 7509 1375 5393 5589 5037 2115 4707 5310 6811 6093 900 7399 2410 1219 6654 3263 6017 3750 5998 4939 5971 4148 3750 803 1985 7194 4780 796 6038 4231 648 1722 6407 3750 1099 6485 1920 1767 5915 6518 6093 5598 5648 4280 900 7326 6242 5328 1214 3870 1985 7194 5998 5741 2115 913 5950 3800 1538 686 6734 6017 3750 1985 3659 1324 5814 4998 5176 535 7399 307 4068 486 1667 1099 2121 6407 3750 7420 3099 6038 4231 4190 1519 3255 7123 4305 3231 1635 4822 1722 3750 2967 3731 1757 1939 648 473 6518 2400 2614 5330 5530 1394 4939 1903 7495 7239 900 4469 5530 4704 299 7467 2121 669 5693 3750 3618 299 5264 4853 1734 316 2828 5445 4190 4939 3484 6043 2376 1031 761 900 5370 3782 2210 669 2210 3099 1363 6301 3508 1907 2410 7509 5718 541 3750 803 2967 6758 3038 6641 1985 7194 512 4811 6811 5243 2112 3750 1734 2376 2891 1211 648 7257 4148 7159 1667 3750 5816 4202 2400 5864 3915 7399 3414 1667 5977 7327 7256 2935 4936 1667 2151 900 6831 4599 6182 3227 3859 3099 7509 7256 3750 1985 7194 4128 4691 2029 1344 6630 5598 1465 648 3706 7403 543 3038 900 1985 7194 3800 980 6017 980 4124 648 900 1635 3605 5028 3731 4109 3792 1866 3578 3915 648 4939 1335 6666 6560 3750 3618 3508 1907 2410 1913 6656 3750 2828 4704 4998 4939 7039 3915 4167 5338 3750 803 1985 4939 3263 7123 264 2456 5689 2109 648 3750 6093 1699 5589 4411 1866 4750 648 1667 1099 3000 7420 1279 2975 1141 7148 3750 1985 3915 2570 4936 5998 1877 3000 7420 900 1635 5470 2313 5864 641 4333 3750 3915 5659 316 2828 2770 5176 803 2047 7532 606 6980 1635 3750 803 1750 7039 3800 7245 3099 7509 5839 3750 1866 1401 4321 5788 1519 6122 6405 4939 5998 2729 900 1985 7194 5998 2289 2107 1519 1592 316 2828 1679 4811 5461 3324 4525 4052 3750 2212 742 3750 1985 7194 6631 1335 5445 3750 1465 7194 4128 6887 4819 5977 3223 2717 900 5612 5948 3750 1985 7194 2289 913 3800 4811 6122 2614 2047 7532 606 6980 900 1985 2541 4409 3772 6012 1833 5560 4173 6662 414 340 316 4125 4128 3800 669 6575 4819 5977 900 1635 25 1460 619 7044 4921 648 4407 3800 1241 600 3750 5470 2313 641 4333 7539 803 316 4125 648 3530 6637 569 1985 3000 4659 5610 6917 3750 3618 1985 6887 7010 3870 900 3915 4939 7010 3870 5598 1985 1394 3397 5598 900 1635 1460 619 5708 1335 6518 4148 3750 2410 1219 6654 2252 1702 5598 803 4646 2109 6905 5520 1635 2663 885 5491 1465 4822 1722 5011 2376 4149 1903 2662 3750 803 316 2828 1767 5915 6065 2042 1335 5598 3750 2688 5598 3231 5780 7399 3750 4811 5788 1292 1641 1667 1099 4811 5393 6407 5708 6631 1335 6666 900 316 4125 4811 648 4939 6678 3750 2021 1726 340 4469 4842 4128 669 5393 4801 3154 3750 5780 7399 669 3915 544 62 5602 1913 5598 3750 3859 6759 4939 4646 1913 900 1635 1767 5915 6065 4464 5814 648 2410 1219 6654 1815 1699 6038 4231 5698 1375 62 307 3750 803 299 5264 1460 316 2828 5445 3750 1985 3414 1667 7509 3223 3750 5998 4939 669 2364 2975 648 900 1985 3038 5938 5168 3770 1667 3750 2717 368 5693 7117 3750 1985 2131 6909 2192 1141 6831 6015 900 3864 7194 1375 5393 1815 1699 1985 5780 7399 5681 3099 5176 3870 5598 3750 1985 3038 3771 6630 7159 1667 900 1635 5659 7377 3166 5445 3750 1793 6666 648 2614 5736 5537 5526 4128 6887 4811 495 6386 900 1465 7194 1767 5659 2410 1219 6654 340 1362 1829 2304 3193 6822 3750 5330 5264 4321 3750 4173 5619 4109 6227 648 5915 6515 4893 5957 6043 3750 5949 4411 5410 1991 4128 826 2490 3193 2602 3750 803 1985 7194 4516 5264 1394 3800 5659 3731 4109 3792 5081 2918 3750 5051 1985 5612 19 3750 3731 4109 3792 5718 7239 3193 6822 900 1635 7377 5736 3750 2205 7305 2620 2042 5192 1745 3605 6887 5278 299 648 5651 7440 1656 3630 1702 3300 7539 803 1985 340 3731 4109 3792 4190 4811 4464 1519 5778 3166 3750 1985 3038 6235 7399 5998 2313 900 1635 25 910 619 4939 1613 248 3193 4741 4893 3750 2967 3731 1757 1939 648 7495 5028 5949 4939 7539 803 4811 2255 3915 3750 1394 4741 900 6887 2255 3915 3750 1394 669 4741 900 1635'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. 数据预处理\n",
    "text = train.iloc[0]['text']\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "new(): invalid data type 'str'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[14], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m text_tensor \u001b[38;5;241m=\u001b[39m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlong\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mTypeError\u001b[0m: new(): invalid data type 'str'"
     ]
    }
   ],
   "source": [
    "text_tensor = torch.tensor(text, dtype=torch.long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. 使用TfidfVectorizer，转换成词向量\n",
    "tfidfvector = TfidfVectorizer(max_features=3000)\n",
    "train_vec = tfidfvector.fit_transform(train['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "sparse array length is ambiguous; use getnnz() or shape[0]",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[6], line 15\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlabel_sequences[idx], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmulti_class_labels[idx]\n\u001b[1;32m     14\u001b[0m \u001b[38;5;66;03m# 创建数据集和数据加载器\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m dataset \u001b[38;5;241m=\u001b[39m \u001b[43mLabelDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrain_vec\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtrain\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlabel\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     16\u001b[0m dataloader \u001b[38;5;241m=\u001b[39m DataLoader(dataset, batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m, shuffle\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
      "Cell \u001b[0;32mIn[6], line 5\u001b[0m, in \u001b[0;36mLabelDataset.__init__\u001b[0;34m(self, label_sequences, multi_class_labels)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, label_sequences, multi_class_labels):\n\u001b[0;32m----> 5\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlabel_sequences \u001b[38;5;241m=\u001b[39m [torch\u001b[38;5;241m.\u001b[39mtensor(seq, dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mlong) \u001b[38;5;28;01mfor\u001b[39;00m seq \u001b[38;5;129;01min\u001b[39;00m label_sequences]\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmulti_class_labels \u001b[38;5;241m=\u001b[39m multi_class_labels\n",
      "Cell \u001b[0;32mIn[6], line 5\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, label_sequences, multi_class_labels):\n\u001b[0;32m----> 5\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlabel_sequences \u001b[38;5;241m=\u001b[39m [\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mseq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlong\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m seq \u001b[38;5;129;01min\u001b[39;00m label_sequences]\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmulti_class_labels \u001b[38;5;241m=\u001b[39m multi_class_labels\n",
      "File \u001b[0;32m~/miniconda3/envs/countVctorRidgeclassification/lib/python3.10/site-packages/scipy/sparse/_base.py:425\u001b[0m, in \u001b[0;36m_spbase.__len__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    424\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__len__\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 425\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msparse array length is ambiguous; use getnnz()\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    426\u001b[0m                     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m or shape[0]\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mTypeError\u001b[0m: sparse array length is ambiguous; use getnnz() or shape[0]"
     ]
    }
   ],
   "source": [
    "# 2. 模型训练 \n",
    "## 2.1 转化成dataloader\n",
    "class LabelDataset(Dataset):\n",
    "    def __init__(self, label_sequences, multi_class_labels):\n",
    "        self.label_sequences = [torch.tensor(seq, dtype=torch.long) for seq in label_sequences]\n",
    "        self.multi_class_labels = multi_class_labels\n",
    " \n",
    "    def __len__(self):\n",
    "        return len(self.label_sequences)\n",
    " \n",
    "    def __getitem__(self, idx):\n",
    "        return self.label_sequences[idx], self.multi_class_labels[idx]\n",
    "\n",
    "# 创建数据集和数据加载器\n",
    "dataset = LabelDataset(train_vec, train['label'])\n",
    "dataloader = DataLoader(dataset, batch_size=2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 2.2 转化成dataloader\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['tf_idf_ridge_classifier_model.joblib']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模型存储\n",
    "joblib.dump(clf, 'tf_idf_ridge_classifier_model.joblib')\n",
    "\n",
    "# 模型加载\n",
    "# loaded_clf = joblib.load('tf_idf_ridge_classifier_model.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8857359438326051"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. 模型评估\n",
    "envalue_pred = clf.predict(train_vec[:10000])\n",
    "envalue_f1_score = f1_score(train['label'][:10000], envalue_pred, average='macro')\n",
    "envalue_f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. 模型预测\n",
    "test = pd.read_csv('/root/data/test_a.csv', sep = '\\t')\n",
    "text_vec = tfidfvector.transform(test['text'])\n",
    "pred = clf.predict(text_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 8, ..., 1, 3, 1])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 5. 结果保存\n",
    "test['label'] = pred\n",
    "test['label'].to_csv(\"/root/data/TFIDF_Ridgeclassification.csv\", index=False)  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "countVctorRidgeclassification",
   "language": "python",
   "name": "countvctorridgeclassification"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
